{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import _pickle as cPickle\n",
    "import json\n",
    "from numpy.random import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 用户和item索引\n",
    "dpath = 'music/Data/'\n",
    "users_index = cPickle.load(open(dpath+'users_index.pkl','rb'))\n",
    "items_index = cPickle.load(open(dpath+'items_index.pkl','rb'))\n",
    "\n",
    "n_users = len(users_index)\n",
    "n_items = len(items_index)\n",
    "\n",
    "# 到排序\n",
    "#每个用户打过分的电影\n",
    "user_items = cPickle.load(open(dpath+'user_items.pkl','rb'))\n",
    "item_users = cPickle.load(open(dpath+'item_users.pkl','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>user</th>\n",
       "      <th>song</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>18707</td>\n",
       "      <td>b4cfdefb94d1df6714c9962923edb73470e6fa7b</td>\n",
       "      <td>SOFCPOU12A8C13BF40</td>\n",
       "      <td>24.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>5704</td>\n",
       "      <td>4bcc4cfd9acf7e19bbccd398f8503ba79fb66513</td>\n",
       "      <td>SOWCKVR12A8C142411</td>\n",
       "      <td>20.714286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>27517</td>\n",
       "      <td>42ac141a65053a2ba02c5380ccf7975022e307a6</td>\n",
       "      <td>SOWOMMY127F8096DF9</td>\n",
       "      <td>56.521739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>14695</td>\n",
       "      <td>625d0167edbc5df88e9fbebe3fcdd6b121a316bb</td>\n",
       "      <td>SOPKPFW12A6D4F84BC</td>\n",
       "      <td>3.448276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>33435</td>\n",
       "      <td>22f6aae94643c2cea285413068f80274e7f1f75e</td>\n",
       "      <td>SONHWUN12AC468C014</td>\n",
       "      <td>29.729730</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                      user                song  \\\n",
       "0       18707  b4cfdefb94d1df6714c9962923edb73470e6fa7b  SOFCPOU12A8C13BF40   \n",
       "1        5704  4bcc4cfd9acf7e19bbccd398f8503ba79fb66513  SOWCKVR12A8C142411   \n",
       "2       27517  42ac141a65053a2ba02c5380ccf7975022e307a6  SOWOMMY127F8096DF9   \n",
       "3       14695  625d0167edbc5df88e9fbebe3fcdd6b121a316bb  SOPKPFW12A6D4F84BC   \n",
       "4       33435  22f6aae94643c2cea285413068f80274e7f1f75e  SONHWUN12AC468C014   \n",
       "\n",
       "       score  \n",
       "0  24.000000  \n",
       "1  20.714286  \n",
       "2  56.521739  \n",
       "3   3.448276  \n",
       "4  29.729730  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u_data = pd.read_csv(dpath+'train.csv')\n",
    "u_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 初始化模型参数\n",
    "# 隐含变量的维数K\n",
    "\n",
    "K = 40\n",
    "\n",
    "#隐含变量的偏置\n",
    "# 初始化维x * 1的向量\n",
    "bu = np.zeros((n_users,1))\n",
    "bi = np.zeros((n_items,1))\n",
    "\n",
    "# item和用户的隐含变量\n",
    "pu = np.zeros((n_users,K))\n",
    "qi = np.zeros((n_items,K))\n",
    "\n",
    "# 用随机变量初始化pu和qi\n",
    "for uid in range(n_users):\n",
    "    pu[uid] = np.reshape(random((1,K))/10*(np.sqrt(K)),K)\n",
    "    \n",
    "for iid in range(n_items):\n",
    "    qi[iid] = np.reshape(random((1,K))/10*(np.sqrt(K)),K)\n",
    "\n",
    "mu = u_data['score'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def svd_pred(uid, iid):\n",
    "    score = mu + bi[iid] + bu[uid] + np.sum(qi[iid] * pu[uid])\n",
    "    \n",
    "    return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The 0 -th step is running\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:29: RuntimeWarning: overflow encountered in square\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:37: RuntimeWarning: overflow encountered in multiply\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/numpy/core/fromnumeric.py:90: RuntimeWarning: invalid value encountered in reduce\n",
      "  return ufunc.reduce(obj, axis, dtype, out, **passkwargs)\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:38: RuntimeWarning: overflow encountered in multiply\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:2: RuntimeWarning: overflow encountered in multiply\n",
      "  \n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:38: RuntimeWarning: invalid value encountered in subtract\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:37: RuntimeWarning: invalid value encountered in add\n",
      "/Users/jay/opt/anaconda3/lib/python3.7/site-packages/numpy/core/fromnumeric.py:90: RuntimeWarning: overflow encountered in reduce\n",
      "  return ufunc.reduce(obj, axis, dtype, out, **passkwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the rmse of this step on train data is [nan]\n",
      "The 1 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 2 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 3 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 4 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 5 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 6 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 7 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 8 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 9 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 10 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 11 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 12 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 13 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 14 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 15 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 16 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 17 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 18 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 19 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 20 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 21 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 22 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 23 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 24 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 25 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 26 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 27 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 28 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 29 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 30 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 31 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 32 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 33 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 34 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 35 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 36 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 37 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 38 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 39 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 40 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 41 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 42 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 43 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 44 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 45 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 46 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 47 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 48 -th step is running\n",
      "the rmse of this step on train data is [nan]\n",
      "The 49 -th step is running\n",
      "the rmse of this step on train data is [nan]\n"
     ]
    }
   ],
   "source": [
    "# 模型训练\n",
    "# gamma 学习率\n",
    "# Lamada 正则参数\n",
    "# steps 迭代次数\n",
    "\n",
    "steps = 50\n",
    "gamma = 0.04\n",
    "Lamada = 0.15\n",
    "\n",
    "# 总的打分记录数目\n",
    "n_records = u_data.shape[0]\n",
    "\n",
    "for step in range(steps):\n",
    "    print('The {} -th step is running'.format(step))\n",
    "    rmse_sum = 0.0\n",
    "    \n",
    "    # 将训练样本打散  获取一个乱序的array\n",
    "    kk = np.random.permutation(n_records)\n",
    "    for j in range(n_records):\n",
    "        line = kk[j]\n",
    "        \n",
    "        uid = users_index[u_data.iloc[line]['user']]\n",
    "        iid = items_index[u_data.iloc[line]['song']]\n",
    "        \n",
    "        rating = u_data.iloc[line]['score']\n",
    "        \n",
    "        # 预测残差\n",
    "        eui = rating - svd_pred(uid,iid)\n",
    "        rmse_sum += eui ** 2\n",
    "        \n",
    "        # 随机梯度下降更新\n",
    "        bu[uid] += gamma * (eui - Lamada * bu[uid])\n",
    "        bi[iid] += gamma * (eui - Lamada * bi[iid])\n",
    "        \n",
    "        # 更新qi pu 加了正则项\n",
    "        temp = qi[iid]\n",
    "        qi[iid] += gamma * (eui * pu[uid] - Lamada * qi[iid])\n",
    "        pu[uid] += gamma * (eui * temp - Lamada * pu[uid])\n",
    "        \n",
    "    # 学习率递减\n",
    "    gamma = gamma * 0.93\n",
    "    print('the rmse of this step on train data is {}'.format(np.sqrt(rmse_sum/n_records)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_json(filepath):\n",
    "    dict_ = {}\n",
    "    dict_['mu'] = mu\n",
    "    dict_['K'] = K\n",
    "    dict_['bi'] = bi.tolist()\n",
    "    dict_['bu'] = bu.tolist()\n",
    "    dict_['qi'] = qi.tolist()\n",
    "    dict_['pu'] = pu.tolist()\n",
    "    json_txt = json.dumps(dict_)\n",
    "    with open(filepath,'w') as file:\n",
    "        file.write(json_txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_json(filepath):\n",
    "    with open(filepath,'r') as file:\n",
    "        dict_ = json.load(file)\n",
    "        \n",
    "        mu = dict_['mu']\n",
    "        K = dict_['K']\n",
    "        \n",
    "        bi = np.asarray(dict_['bi'])\n",
    "        bu = np.asarray(dict_['bu'])\n",
    "        \n",
    "        qi = np.asarray(dict_['qi'])\n",
    "        pu = np.asarray(dict_['pu'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_json(dpath+'svd_model.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "load_json(dpath+'svd_model.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 对物品进行推荐\n",
    "\n",
    "def recommend(user):\n",
    "    cur_user_id = users_index[user]\n",
    "    \n",
    "    # 训练集中用户打过分的item\n",
    "    cur_user_items = user_items[cur_user_id]\n",
    "    \n",
    "    # 该用户对所有物品打分\n",
    "    user_item_scores = np.zeros(n_items)\n",
    "    \n",
    "    # 预测打分\n",
    "    for i in range(n_items):\n",
    "        if i not in cur_user_items:\n",
    "            user_item_scores[i] = svd_pred(cur_user_id,i)\n",
    "    # 用元组来存（分数，物品id）\n",
    "    sort_index = sorted(((e,i) for i,e in enumerate(list(user_item_scores))),reverse=True)\n",
    "    columns = ['song','score']\n",
    "    df = pd.DataFrame(columns=columns)\n",
    "    \n",
    "    for i in range(0,len(sort_index)):\n",
    "        cur_item_index = sort_index[i][1]\n",
    "        # 把index转化乘list然后通过index定位value所在位置，然后再将key（物品真正的id）转化成list，找到真正的item id\n",
    "        cur_item = list(items_index.keys())[list(items_index.values()).index(cur_item_index)]\n",
    "        \n",
    "        if ~np.isnan(sort_index[i][0]) and cur_item_index not in cur_user_items:\n",
    "            df.loc[len(df)] = [cur_item,sort_index[i][0]]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>user</th>\n",
       "      <th>song</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>30628</td>\n",
       "      <td>b7c24f770be6b802805ac0e2106624a517643c17</td>\n",
       "      <td>SOEBOWM12AB017F279</td>\n",
       "      <td>10.810811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>31485</td>\n",
       "      <td>9254a3fdc569428c3b1c3904db36d485c47e2544</td>\n",
       "      <td>SOPXKYD12A6D4FA876</td>\n",
       "      <td>23.076923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>16920</td>\n",
       "      <td>31aad1036a404737ee8b88ea2da68813c9a46874</td>\n",
       "      <td>SOQJHUW12AB0188A24</td>\n",
       "      <td>2.678571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>16954</td>\n",
       "      <td>e3e8103d0751e29693f9b03a58efa5c21acf2115</td>\n",
       "      <td>SODJWHY12A8C142CCE</td>\n",
       "      <td>17.241379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>25098</td>\n",
       "      <td>520bb6f7bd2fc51d02f236398acdc5170cc299a8</td>\n",
       "      <td>SOWNVIV12AB0184846</td>\n",
       "      <td>12.500000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                      user                song  \\\n",
       "0       30628  b7c24f770be6b802805ac0e2106624a517643c17  SOEBOWM12AB017F279   \n",
       "1       31485  9254a3fdc569428c3b1c3904db36d485c47e2544  SOPXKYD12A6D4FA876   \n",
       "2       16920  31aad1036a404737ee8b88ea2da68813c9a46874  SOQJHUW12AB0188A24   \n",
       "3       16954  e3e8103d0751e29693f9b03a58efa5c21acf2115  SODJWHY12A8C142CCE   \n",
       "4       25098  520bb6f7bd2fc51d02f236398acdc5170cc299a8  SOWNVIV12AB0184846   \n",
       "\n",
       "       score  \n",
       "0  10.810811  \n",
       "1  23.076923  \n",
       "2   2.678571  \n",
       "3  17.241379  \n",
       "4  12.500000  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 测试\n",
    "df_triplet_test = pd.read_csv(dpath+'test.csv')\n",
    "df_triplet_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6b3d5eaba2e55699cb725d0c605c6eca1b302dfe is new user\n",
      "a3a9329463c55f63876f84b0c47b4f90ca9db7bc is new user\n",
      "de27b74444dae039f76e421362c6a914da9f8b41 is new user\n",
      "6a58f480d522814c087fd3f8c77b3f32bb161f9d is new user\n",
      "52a6c7b6221f57c89dacbbd06854ca0dc415e9e6 is new user\n",
      "467e0e46181933c7e1a936e513ca55fbab4edaed is new user\n"
     ]
    }
   ],
   "source": [
    "# 统计总的用户\n",
    "unique_users_test = df_triplet_test['user'].unique()\n",
    "\n",
    "# 为每个用户推荐10个商品\n",
    "n_rec_items = 10\n",
    "\n",
    "#性能评价计算精确率和召回率\n",
    "n_hits = 0\n",
    "n_total_rec_items = 0\n",
    "n_test_items = 0\n",
    "\n",
    "#所有被推荐商品的集合，用于计算覆盖度\n",
    "all_rec_items = set()\n",
    "\n",
    "# 残差平方和，用于计算RMSE\n",
    "rss_test = 0.0\n",
    "\n",
    "# 对每个测试用户\n",
    "for user in unique_users_test:\n",
    "    if user not in users_index:\n",
    "        print('{} is new user'.format(user))\n",
    "        continue\n",
    "    user_records_test = df_triplet_test[df_triplet_test.user == user]\n",
    "    \n",
    "    rec_items = recommend(user)\n",
    "    \n",
    "    for i in range(n_rec_items):\n",
    "        item = rec_items.iloc[i]['song']\n",
    "        \n",
    "        if item in user_records_test['song'].values:\n",
    "            n_hits +=1\n",
    "            \n",
    "        all_rec_items.add(item)\n",
    "    \n",
    "    # 计算rmse\n",
    "    for i in range(user_records_test.shape[0]):\n",
    "        item = user_records_test.iloc[i]['song']\n",
    "        score = user_records_test.iloc[i]['score']\n",
    "        \n",
    "        df1 = rec_items[rec_items.song == item]\n",
    "        if df1.shape[0] == 0:\n",
    "            print('{} is new item. '.format(item))\n",
    "            continue\n",
    "        pre_score = df1['score'].values[0]\n",
    "        rss_test += (pre_score - score)**2\n",
    "    # 推荐item 总数\n",
    "    n_total_rec_items += n_rec_items\n",
    "    n_test_items += user_records_test.shape[0]\n",
    "\n",
    "precision = n_hits / (1.0 * n_total_rec_items)\n",
    "recall = n_hits / (1.0 * n_test_items)\n",
    "\n",
    "# 覆盖率\n",
    "coverage = len(all_rec_items) / (1.0 * n_items)\n",
    "\n",
    "# 打分均方误差\n",
    "rmse = np.sqrt(rss_test/df_triplet_test.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.013067400275103164"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.012670045345425446"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.23875"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coverage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22.707294497811432"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rmse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
